import matplotlib
matplotlib.use('TkAgg')  # 在导入 gradio 之前设置后端

import gradio

import gradio as gr
import torch
from transformers import AutoConfig, AutoModelForCausalLM
from janus.models import MultiModalityCausalLM, VLChatProcessor
from janus.utils.io import load_pil_images
from PIL import Image

import numpy as np
import os
import time
# import spaces  # Import spaces for ZeroGPU compatibility


# Load model and processor
# model_path = "deepseek-ai/Janus-Pro-7B"
model_path = "deepseek-ai/Janus-Pro-1B"
config = AutoConfig.from_pretrained(model_path)
language_config = config.language_config
language_config._attn_implementation = 'eager'
vl_gpt = AutoModelForCausalLM.from_pretrained(model_path,
                                             language_config=language_config,
                                             trust_remote_code=True)
if torch.cuda.is_available():
    vl_gpt = vl_gpt.to(torch.bfloat16).cuda()
else:
    # vl_gpt = vl_gpt.to(torch.float16)
    vl_gpt = vl_gpt.to(torch.float32)

vl_chat_processor = VLChatProcessor.from_pretrained(model_path)
tokenizer = vl_chat_processor.tokenizer
cuda_device = 'cuda' if torch.cuda.is_available() else 'cpu'

@torch.inference_mode()
# @spaces.GPU(duration=120) 
# Multimodal Understanding function
def multimodal_understanding(image, question, seed, top_p, temperature):
    # Clear CUDA cache before generating
    torch.cuda.empty_cache()
    
    # set seed
    torch.manual_seed(seed)
    np.random.seed(seed)
    torch.cuda.manual_seed(seed)
    
    conversation = [
        {
            "role": "<|User|>",
            "content": f"<image_placeholder>\n{question}",
            "images": [image],
        },
        {"role": "<|Assistant|>", "content": ""},
    ]
    
    pil_images = [Image.fromarray(image)]
    # prepare_inputs = vl_chat_processor(
    #     conversations=conversation, images=pil_images, force_batchify=True
    # ).to(cuda_device, dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float16)
    # 修改prepare_inputs部分
    prepare_inputs = vl_chat_processor(
        conversations=conversation, 
        images=pil_images, 
        force_batchify=True
    ).to(cuda_device, dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32)

    
    inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
    
    outputs = vl_gpt.language_model.generate(
        inputs_embeds=inputs_embeds,
        attention_mask=prepare_inputs.attention_mask,
        pad_token_id=tokenizer.eos_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id,
        max_new_tokens=512,
        do_sample=False if temperature == 0 else True,
        use_cache=True,
        temperature=temperature,
        top_p=top_p,
    )
    
    answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
    return answer


def generate(input_ids,
             width,
             height,
             temperature: float = 1,
             parallel_size: int = 5,
             cfg_weight: float = 5,
             image_token_num_per_image: int = 576,
             patch_size: int = 16):
    # Clear CUDA cache before generating
    torch.cuda.empty_cache()
    
    tokens = torch.zeros((parallel_size * 2, len(input_ids)), dtype=torch.int).to(cuda_device)
    for i in range(parallel_size * 2):
        tokens[i, :] = input_ids
        if i % 2 != 0:
            tokens[i, 1:-1] = vl_chat_processor.pad_id
    inputs_embeds = vl_gpt.language_model.get_input_embeddings()(tokens)
    generated_tokens = torch.zeros((parallel_size, image_token_num_per_image), dtype=torch.int).to(cuda_device)

    pkv = None
    for i in range(image_token_num_per_image):
        with torch.no_grad():
            outputs = vl_gpt.language_model.model(inputs_embeds=inputs_embeds,
                                                use_cache=True,
                                                past_key_values=pkv)
            pkv = outputs.past_key_values
            hidden_states = outputs.last_hidden_state
            logits = vl_gpt.gen_head(hidden_states[:, -1, :])
            logit_cond = logits[0::2, :]
            logit_uncond = logits[1::2, :]
            logits = logit_uncond + cfg_weight * (logit_cond - logit_uncond)
            probs = torch.softmax(logits / temperature, dim=-1)
            next_token = torch.multinomial(probs, num_samples=1)
            generated_tokens[:, i] = next_token.squeeze(dim=-1)
            next_token = torch.cat([next_token.unsqueeze(dim=1), next_token.unsqueeze(dim=1)], dim=1).view(-1)

            img_embeds = vl_gpt.prepare_gen_img_embeds(next_token)
            inputs_embeds = img_embeds.unsqueeze(dim=1)

    

    patches = vl_gpt.gen_vision_model.decode_code(generated_tokens.to(dtype=torch.int),
                                                 shape=[parallel_size, 8, width // patch_size, height // patch_size])

    return generated_tokens.to(dtype=torch.int), patches

def unpack(dec, width, height, parallel_size=5):
    dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1)
    dec = np.clip((dec + 1) / 2 * 255, 0, 255)

    visual_img = np.zeros((parallel_size, width, height, 3), dtype=np.uint8)
    visual_img[:, :, :] = dec

    return visual_img



@torch.inference_mode()
# @spaces.GPU(duration=120)  # Specify a duration to avoid timeout
def generate_image(prompt,
                   seed=None,
                   guidance=5,
                   t2i_temperature=1.0):
    # Clear CUDA cache and avoid tracking gradients
    torch.cuda.empty_cache()
    # Set the seed for reproducible results
    if seed is not None:
        torch.manual_seed(seed)
        torch.cuda.manual_seed(seed)
        np.random.seed(seed)
    width = 384
    height = 384
    parallel_size = 5
    
    with torch.no_grad():
        messages = [{'role': '<|User|>', 'content': prompt},
                    {'role': '<|Assistant|>', 'content': ''}]
        text = vl_chat_processor.apply_sft_template_for_multi_turn_prompts(conversations=messages,
                                                                   sft_format=vl_chat_processor.sft_format,
                                                                   system_prompt='')
        text = text + vl_chat_processor.image_start_tag
        
        input_ids = torch.LongTensor(tokenizer.encode(text))
        output, patches = generate(input_ids,
                                   width // 16 * 16,
                                   height // 16 * 16,
                                   cfg_weight=guidance,
                                   parallel_size=parallel_size,
                                   temperature=t2i_temperature)
        images = unpack(patches,
                        width // 16 * 16,
                        height // 16 * 16,
                        parallel_size=parallel_size)

        return [Image.fromarray(images[i]).resize((768, 768), Image.LANCZOS) for i in range(parallel_size)]
        

# Gradio interface
# with gr.Blocks() as demo:
#     gr.Markdown(value="# Multimodal Understanding")
#     with gr.Row():
#         image_input = gr.Image()
#         with gr.Column():
#             question_input = gr.Textbox(label="Question")
#             und_seed_input = gr.Number(label="Seed", precision=0, value=42)
#             top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="top_p")
#             temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="temperature")
#
#     understanding_button = gr.Button("Chat")
#     understanding_output = gr.Textbox(label="Response")
#
#     examples_inpainting = gr.Examples(
#         label="Multimodal Understanding examples",
#         examples=[
#             [
#                 "explain this meme",
#                 "images/doge.png",
#             ],
#             [
#                 "Convert the formula into latex code.",
#                 "images/equation.png",
#             ],
#         ],
#         inputs=[question_input, image_input],
#     )
#
#
#     gr.Markdown(value="# Text-to-Image Generation")
#
#
#
#     with gr.Row():
#         cfg_weight_input = gr.Slider(minimum=1, maximum=10, value=5, step=0.5, label="CFG Weight")
#         t2i_temperature = gr.Slider(minimum=0, maximum=1, value=1.0, step=0.05, label="temperature")
#
#     prompt_input = gr.Textbox(label="Prompt. (Prompt in more detail can help produce better images!)")
#     seed_input = gr.Number(label="Seed (Optional)", precision=0, value=12345)
#
#     generation_button = gr.Button("Generate Images")
#
#     image_output = gr.Gallery(label="Generated Images", columns=2, rows=2, height=300)
#
#     examples_t2i = gr.Examples(
#         label="Text to image generation examples.",
#         examples=[
#             "Master shifu racoon wearing drip attire as a street gangster.",
#             "The face of a beautiful girl",
#             "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k",
#             "A glass of red wine on a reflective surface.",
#             "A cute and adorable baby fox with big brown eyes, autumn leaves in the background enchanting,immortal,fluffy, shiny mane,Petals,fairyism,unreal engine 5 and Octane Render,highly detailed, photorealistic, cinematic, natural colors.",
#             "The image features an intricately designed eye set against a circular backdrop adorned with ornate swirl patterns that evoke both realism and surrealism. At the center of attention is a strikingly vivid blue iris surrounded by delicate veins radiating outward from the pupil to create depth and intensity. The eyelashes are long and dark, casting subtle shadows on the skin around them which appears smooth yet slightly textured as if aged or weathered over time.\n\nAbove the eye, there's a stone-like structure resembling part of classical architecture, adding layers of mystery and timeless elegance to the composition. This architectural element contrasts sharply but harmoniously with the organic curves surrounding it. Below the eye lies another decorative motif reminiscent of baroque artistry, further enhancing the overall sense of eternity encapsulated within each meticulously crafted detail. \n\nOverall, the atmosphere exudes a mysterious aura intertwined seamlessly with elements suggesting timelessness, achieved through the juxtaposition of realistic textures and surreal artistic flourishes. Each component\u2014from the intricate designs framing the eye to the ancient-looking stone piece above\u2014contributes uniquely towards creating a visually captivating tableau imbued with enigmatic allure.",
#         ],
#         inputs=prompt_input,
#     )
#
#     understanding_button.click(
#         multimodal_understanding,
#         inputs=[image_input, question_input, und_seed_input, top_p, temperature],
#         outputs=understanding_output
#     )
#
#     generation_button.click(
#         fn=generate_image,
#         inputs=[prompt_input, seed_input, cfg_weight_input, t2i_temperature],
#         outputs=image_output
#     )
#
# demo.launch(share=True)
# demo.queue(concurrency_count=1, max_size=10).launch(server_name="0.0.0.0", server_port=37906, root_path="/path")

# Gradio界面
with gr.Blocks() as demo:
    gr.Markdown(value="# 多模态理解")
    with gr.Row():
        image_input = gr.Image()
        with gr.Column():
            question_input = gr.Textbox(label="问题")
            und_seed_input = gr.Number(label="种子", precision=0, value=42)
            top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="核采样概率阈值")
            temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="温度")

    understanding_button = gr.Button("聊天")
    understanding_output = gr.Textbox(label="回复")

    examples_inpainting = gr.Examples(
        label="多模态理解示例",
        examples=[
            [
                "解释这个表情包",
                # "images/doge.png",
                "doge.png",
            ],
            [
                "将公式转换为latex代码。",
                # "images/equation.png",
                "equation.png",
            ],
        ],
        inputs=[question_input, image_input],
    )

    gr.Markdown(value="# 文本到图像生成")

    with gr.Row():
        cfg_weight_input = gr.Slider(minimum=1, maximum=10, value=5, step=0.5, label="CFG 权重")
        t2i_temperature = gr.Slider(minimum=0, maximum=1, value=1.0, step=0.05, label="温度")

    prompt_input = gr.Textbox(label="提示词。（更详细的提示词可以帮助生成更好的图像！）")
    seed_input = gr.Number(label="种子（可选）", precision=0, value=12345)

    generation_button = gr.Button("生成图像")

    image_output = gr.Gallery(label="生成的图像", columns=2, rows=2, height=300)

    examples_t2i = gr.Examples(
        label="文本到图像生成示例。",
        examples=[
            "街头黑帮装扮的师父浣熊。",
            "美丽女孩的脸",
            "丛林中的宇航员，冷色调，柔和色彩，细节丰富，8k",
            "反射表面上的一杯红酒。",
            "一只可爱迷人的小狐狸，棕色大眼睛，秋天的背景，迷人、永恒、蓬松、闪亮的鬃毛，花瓣，仙女气息，虚幻引擎5与Octane Render，高度细节，写实，电影感，天然色彩。",
            "这幅图描绘了一个复杂设计的眼睛，背景是一个圆形的装饰性图案，既具有现实主义风格又带有超现实主义的元素。眼睛的中央是一只生动的蓝色虹膜，周围是精细的血管，向外辐射，创造深度和强烈的效果。睫毛长而深色，在周围的皮肤上投下微妙的阴影，皮肤看起来光滑却略微有些质感，仿佛经过时间的雕刻或风化。\n\n眼睛上方是一个像石材一样的结构，像是古典建筑的一部分，增添了神秘和永恒的优雅。这个建筑元素与周围有机曲线形成了鲜明的对比，同时又和谐地融合在一起。眼睛下方有一个装饰性图案，像是巴洛克艺术风格，进一步增强了这幅作品的永恒感。\n\n总体来说，图像给人一种神秘的气息，融入了现实与超现实艺术元素的完美融合。每个组件—从框架眼睛的复杂设计到古老的石质结构—都为创造一个视觉上引人注目的画面贡献了独特的元素。",
        ],
        inputs=prompt_input,
    )

    understanding_button.click(
        multimodal_understanding,
        inputs=[image_input, question_input, und_seed_input, top_p, temperature],
        outputs=understanding_output
    )

    generation_button.click(
        fn=generate_image,
        inputs=[prompt_input, seed_input, cfg_weight_input, t2i_temperature],
        outputs=image_output
    )

demo.launch(share=True)
